"""""""""
Final project for 5242 : Exploration on GAN
Group 38
By:
Chengming Xie
Jiarui Yang
Xinyi Wang
""""""""""
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import scipy.io as spio
from PIL import Image
from zipfile import ZipFile
import time
import tensorflow as tf
import math
def load_mnist(activation = "tanh", BATCH_SIZE = 64):
BUFFER_SIZE = 10000
(train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()
train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')
if activation == "tanh":
train_x = (train_images - 127.5)/127.5
else:
train_x = train_images/255.0
print("training data shape", train_x.shape)
train_dataset = tf.data.Dataset.from_tensor_slices(train_x).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
return train_dataset, train_x
def load_training_data(mat_file = "train_32x32.mat",activation = "tanh", BATCH_SIZE = 64,
test_plot = True):
BUFFER_SIZE = 10000
train = spio.loadmat("train_32x32.mat")
train_x,train_y = train['X'],train['y']
train_x = np.moveaxis(train_x,3,0).astype('float32')
if activation == "tanh":
train_x = (train_x - 127.5)/127.5
else:
train_x = train_x/255.0
print("training data shape", train_x.shape)
train_dataset = tf.data.Dataset.from_tensor_slices(train_x).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
return train_dataset, train_x
def load_celeb_data(file_name = "100k.zip",BATCH_SIZE = 64,
sample_num = 10000, gen_plot = True):
BUFFER_SIZE = 10000
with ZipFile(file_name, 'r') as zip_:
zip_.extractall()
output_images = []
path_celeb = "/content/100k/"
for path in os.listdir(path_celeb):
if '.jpg' in path:
output_images.append(os.path.join(path_celeb, path))
data=output_images[0:sample_num]
images = [np.array((Image.open(path)).resize((64,64))) for path in data]
for i in range(len(images)):
images[i] = images[i]/255
train_x = np.array(images).reshape(-1,64,64,3)
print("training data shape is ", train_x.shape)
if gen_plot:
print("5 real images in the dataset")
for i in range(5):
plt.subplot(1,5,i+1)
plt.xticks([])
plt.yticks([])
plt.imshow(train_x[i], cmap = plt.cm.binary)
plt.show()
train_dataset = tf.data.Dataset.from_tensor_slices(train_x).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
return train_dataset, train_x
def make_discriminator_model(layer_config,size = (64,32,32,3),
activation = "sigmoid",add_dropout = False):
input_shape = list(size[1:])
model = tf.keras.Sequential()
for num_layer,layer in enumerate(layer_config):
Units,Kernel,Strides,Padding,Batch_norm,Activation= list(layer_config[layer].values())
if num_layer == 0:
model.add(tf.keras.layers.Conv2D(Units, Kernel, strides=Strides, padding=Padding,
input_shape=input_shape))
else:
model.add(tf.keras.layers.Conv2D(Units, Kernel, strides=Strides, padding=Padding))
if Batch_norm:
model.add(tf.keras.layers.BatchNormalization())
model.add(Activation)
model.add(tf.keras.layers.Flatten())
if add_dropout:
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Dense(1,activation = activation))
model.summary()
return model
def make_generator_model(layer_config,size = (64,32,32,3),test = True,dataset = "MNIST"):
if dataset == "SVNH":
input_dim = size[1]//8
elif dataset == "MNIST":
input_dim = size[1]//4
elif dataset == "celeb":
input_dim = size[1]//16
model = tf.keras.models.Sequential()
for layer in layer_config:
if layer == 'Dense':
Units,input_shape,use_bias,Batch_norm,Activation,last_layer= list(layer_config[layer].values())
model.add(tf.keras.layers.Dense(input_dim*input_dim*Units,
input_shape=input_shape,use_bias=use_bias))
model.add(tf.keras.layers.Reshape((input_dim,input_dim,Units)))
else:
Units,Kernel,Strides,Padding,use_bias,Batch_norm,Activation,last_layer= list(layer_config[layer].values())
if last_layer:
model.add(tf.keras.layers.Conv2DTranspose(Units, Kernel, strides=Strides, padding=Padding,
use_bias=use_bias,activation = Activation))
else:
model.add(tf.keras.layers.Conv2DTranspose(Units, Kernel, strides=Strides, padding=Padding,use_bias=use_bias))
if not last_layer:
if Batch_norm: model.add(tf.keras.layers.BatchNormalization())
model.add(Activation)
# if layer == "Dense":
# model.add(tf.keras.layers.Reshape((input_dim,input_dim,Units)))
model.summary()
return model
def wasserstein_loss(y_label, y_pred):
return -tf.keras.backend.mean(y_label * y_pred)
cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)
def discriminator_loss(real_output, fake_output):
real_loss = cross_entropy(tf.ones_like(real_output), real_output)
fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)
total_loss = real_loss + fake_loss
return total_loss
def WGAN_discriminator_loss(real_output, fake_output):
real_loss = wasserstein_loss(tf.ones_like(real_output), real_output)
fake_loss = wasserstein_loss(tf.ones_like(fake_output)*(-1), fake_output)
return (real_loss,fake_loss)
def generator_loss(fake_output):
return cross_entropy(tf.ones_like(fake_output), fake_output)
def optimizer(opt = "Adam",learning_rate = 1e-4):
if opt == "Adam":
generator_optimizer = tf.keras.optimizers.Adam(learning_rate)
discriminator_optimizer = tf.keras.optimizers.Adam(learning_rate)
elif opt == "RMSProp":
generator_optimizer = tf.keras.optimizers.RMSprop(learning_rate)
discriminator_optimizer = tf.keras.optimizers.RMSprop(learning_rate)
return generator_optimizer,discriminator_optimizer
@tf.function
def train_step(images,BATCH_SIZE = 64,noise_dim = 100):
noise = tf.random.normal([BATCH_SIZE, noise_dim])
with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
generated_images = generator(noise, training=True)
real_output = discriminator(images, training=True)
fake_output = discriminator(generated_images, training=True)
gen_loss = generator_loss(fake_output)
disc_loss = discriminator_loss(real_output, fake_output)
gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)
gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)
generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))
discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))
@tf.function
def WGAN_train_step(images, n_critic, clip_value,BATCH_SIZE = 64,noise_dim = 100):
## Train Discriminator multiple times before training Generator
for n_iter in range(n_critic):
noise = tf.random.normal([BATCH_SIZE, noise_dim])
for l in range(2):
with tf.GradientTape() as disc_tape,tf.GradientTape() as gen_tape:
generated_images = generator(noise, training=True)
real_output = discriminator(images, training=True)
fake_output = discriminator(generated_images, training=True)
loss = WGAN_discriminator_loss(real_output, fake_output)
# train the discriminator network
# real data label=1, fake data label=-1
# instead of 1 combined batch of real and fake images,
# train with 1 batch of real data first, then 1 batch
# of fake images.
cur_loss = loss[l]
gen_loss = generator_loss(fake_output)
gradients_of_discriminator = disc_tape.gradient(cur_loss, discriminator.trainable_variables)
## Apply gradient clipping to satisfy Lipschitz constraint
gradients_of_discriminator = [tf.clip_by_value(gradient,
clip_value_min=-clip_value,
clip_value_max=clip_value) for gradient in gradients_of_discriminator]
discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator,
discriminator.trainable_variables))
gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)
generator_optimizer.apply_gradients(zip(gradients_of_generator,generator.trainable_variables))
def generate_and_save_images(model, epoch, test_input,activation = "tanh",dataset = "MNIST"):
predictions = model(test_input, training=False)
fig = plt.figure(figsize=(4,4))
rows = int(math.sqrt(predictions.shape[0]))
for i in range(predictions.shape[0]):
plt.subplot(rows, rows, i + 1)
if dataset == "MNIST":
if activation == "tanh":
plt.imshow((predictions[i, :, :, 0]* 127.5 + 127.5).numpy().astype(np.uint8), cmap='binary')
elif activation == "sigmoid":
plt.imshow((predictions[i, :, :, 0]*255).numpy().astype(np.uint8), cmap='binary')
else:
if activation == "tanh":
plt.imshow((predictions[i, :, :, :]* 127.5 + 127.5).numpy().astype(np.uint8), cmap='binary')
elif activation == "sigmoid":
plt.imshow((predictions[i, :, :, :]*255).numpy().astype(np.uint8), cmap='binary')
plt.axis('off')
plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))
plt.show()
def train(dataset, epochs = 50 ,noise_dim = 100, num_examples_to_generate = 16,GAN_type = "Regular",
n_critic = 5,clip_value = 0.01,image_dataset = "MNIST",activation = "tanh"):
seed = tf.random.normal([num_examples_to_generate, noise_dim])
start = time.time()
for epoch in range(epochs):
for image_batch in dataset:
if GAN_type == "Regular":
train_step(image_batch)
elif GAN_type == "WGAN":
WGAN_train_step(image_batch, n_critic = n_critic, clip_value = clip_value)
if epoch % 5 == 0:
generate_and_save_images(generator,
epoch + 1,
seed,
dataset = image_dataset,
activation = activation
)
print ('Time for epoch {} is {} sec'.format(epoch, time.time()-start))
"""""""""
GAN on SVNH dataset
Adam Optimizer with 1e-4 learning rate
Sigmoid activation on generator's last Transpose Conv2D layer
Architectures as two config dictionary
Train for 120 epochs
"""""""""
Generator_config = {
"Dense":{"Units":512,"input_shape": (100,),"use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_1":{"Units": 256,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_2":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_3":{"Units": 3,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias": False,
"Batch_norm":False,"Activation":"sigmoid","last_layer":True}
}
train_dataset,train_x = load_training_data(activation = "sigmoid")
discriminator = make_discriminator_model(layer_config = Discriminator_config,activation = "sigmoid")
generator = make_generator_model(layer_config = Generator_config, dataset = "SVNH")
generator_optimizer,discriminator_optimizer = optimizer("Adam")
train(train_dataset,epochs = 120,GAN_type = "Regular",image_dataset = "SVNH",activation = "sigmoid")
training data shape (73257, 32, 32, 3) Model: "sequential_2" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_3 (Conv2D) (None, 16, 16, 64) 4864 _________________________________________________________________ leaky_re_lu_12 (LeakyReLU) (None, 16, 16, 64) 0 _________________________________________________________________ conv2d_4 (Conv2D) (None, 8, 8, 128) 204928 _________________________________________________________________ batch_normalization_5 (Batch (None, 8, 8, 128) 512 _________________________________________________________________ leaky_re_lu_13 (LeakyReLU) (None, 8, 8, 128) 0 _________________________________________________________________ conv2d_5 (Conv2D) (None, 4, 4, 256) 819456 _________________________________________________________________ batch_normalization_6 (Batch (None, 4, 4, 256) 1024 _________________________________________________________________ leaky_re_lu_14 (LeakyReLU) (None, 4, 4, 256) 0 _________________________________________________________________ flatten_1 (Flatten) (None, 4096) 0 _________________________________________________________________ dense_2 (Dense) (None, 1) 4097 ================================================================= Total params: 1,034,881 Trainable params: 1,034,113 Non-trainable params: 768 _________________________________________________________________ Model: "sequential_3" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_3 (Dense) (None, 8192) 819200 _________________________________________________________________ reshape_1 (Reshape) (None, 4, 4, 512) 0 _________________________________________________________________ batch_normalization_7 (Batch (None, 4, 4, 512) 2048 _________________________________________________________________ leaky_re_lu_9 (LeakyReLU) (None, 4, 4, 512) 0 _________________________________________________________________ conv2d_transpose_3 (Conv2DTr (None, 8, 8, 256) 3276800 _________________________________________________________________ batch_normalization_8 (Batch (None, 8, 8, 256) 1024 _________________________________________________________________ leaky_re_lu_10 (LeakyReLU) (None, 8, 8, 256) 0 _________________________________________________________________ conv2d_transpose_4 (Conv2DTr (None, 16, 16, 128) 819200 _________________________________________________________________ batch_normalization_9 (Batch (None, 16, 16, 128) 512 _________________________________________________________________ leaky_re_lu_11 (LeakyReLU) (None, 16, 16, 128) 0 _________________________________________________________________ conv2d_transpose_5 (Conv2DTr (None, 32, 32, 3) 9600 ================================================================= Total params: 4,928,384 Trainable params: 4,926,592 Non-trainable params: 1,792 _________________________________________________________________
Time for epoch 0 is 24.50974130630493 sec
Time for epoch 5 is 133.5604977607727 sec
Time for epoch 10 is 242.59518098831177 sec
Time for epoch 15 is 351.7689278125763 sec
Time for epoch 20 is 460.79391169548035 sec
Time for epoch 25 is 569.8202042579651 sec
Time for epoch 30 is 678.9619159698486 sec
Time for epoch 35 is 787.965784072876 sec
Time for epoch 40 is 897.1072475910187 sec
Time for epoch 45 is 1006.0312025547028 sec
Time for epoch 50 is 1115.0720698833466 sec
Time for epoch 55 is 1224.242957830429 sec
Time for epoch 60 is 1333.2583804130554 sec
Time for epoch 65 is 1442.2805597782135 sec
Time for epoch 70 is 1551.4423913955688 sec
Time for epoch 75 is 1660.4815833568573 sec
Time for epoch 80 is 1769.4880347251892 sec
Time for epoch 85 is 1878.6809866428375 sec
Time for epoch 90 is 1987.7235243320465 sec
Time for epoch 95 is 2096.7524251937866 sec
Time for epoch 100 is 2205.919884443283 sec
Time for epoch 105 is 2314.9431529045105 sec
Time for epoch 110 is 2423.9495816230774 sec
Time for epoch 115 is 2533.117336034775 sec
"""""""""
GAN on SVNH dataset
Adam Optimizer with 1e-4 learning rate
tanh activation on generator's last Transpose Conv2D layer
Architectures as two config dictionary
Train for 120 epochs
"""""""""
Discriminator_config = {
"CNN_1":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":False,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_2":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_3":{"Units": 256,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)}
}
Generator_config = {
"Dense":{"Units":512,"input_shape": (100,),"use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_1":{"Units": 256,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_2":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_3":{"Units": 3,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias": False,
"Batch_norm":False,"Activation":"tanh","last_layer":True}
}
train_dataset,train_x = load_training_data()
discriminator = make_discriminator_model(layer_config = Discriminator_config,activation = "sigmoid")
generator = make_generator_model(layer_config = Generator_config, dataset = "SVNH")
generator_optimizer,discriminator_optimizer = optimizer("Adam")
train(train_dataset,epochs = 120,GAN_type = "Regular",image_dataset = "SVNH")
training data shape (73257, 32, 32, 3) Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d (Conv2D) (None, 16, 16, 64) 4864 _________________________________________________________________ leaky_re_lu_6 (LeakyReLU) (None, 16, 16, 64) 0 _________________________________________________________________ conv2d_1 (Conv2D) (None, 8, 8, 128) 204928 _________________________________________________________________ batch_normalization (BatchNo (None, 8, 8, 128) 512 _________________________________________________________________ leaky_re_lu_7 (LeakyReLU) (None, 8, 8, 128) 0 _________________________________________________________________ conv2d_2 (Conv2D) (None, 4, 4, 256) 819456 _________________________________________________________________ batch_normalization_1 (Batch (None, 4, 4, 256) 1024 _________________________________________________________________ leaky_re_lu_8 (LeakyReLU) (None, 4, 4, 256) 0 _________________________________________________________________ flatten (Flatten) (None, 4096) 0 _________________________________________________________________ dense (Dense) (None, 1) 4097 ================================================================= Total params: 1,034,881 Trainable params: 1,034,113 Non-trainable params: 768 _________________________________________________________________ Model: "sequential_1" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_1 (Dense) (None, 8192) 819200 _________________________________________________________________ reshape (Reshape) (None, 4, 4, 512) 0 _________________________________________________________________ batch_normalization_2 (Batch (None, 4, 4, 512) 2048 _________________________________________________________________ leaky_re_lu_3 (LeakyReLU) (None, 4, 4, 512) 0 _________________________________________________________________ conv2d_transpose (Conv2DTran (None, 8, 8, 256) 3276800 _________________________________________________________________ batch_normalization_3 (Batch (None, 8, 8, 256) 1024 _________________________________________________________________ leaky_re_lu_4 (LeakyReLU) (None, 8, 8, 256) 0 _________________________________________________________________ conv2d_transpose_1 (Conv2DTr (None, 16, 16, 128) 819200 _________________________________________________________________ batch_normalization_4 (Batch (None, 16, 16, 128) 512 _________________________________________________________________ leaky_re_lu_5 (LeakyReLU) (None, 16, 16, 128) 0 _________________________________________________________________ conv2d_transpose_2 (Conv2DTr (None, 32, 32, 3) 9600 ================================================================= Total params: 4,928,384 Trainable params: 4,926,592 Non-trainable params: 1,792 _________________________________________________________________
Time for epoch 0 is 26.2413649559021 sec
Time for epoch 5 is 135.8301227092743 sec
Time for epoch 10 is 245.1153256893158 sec
Time for epoch 15 is 354.38807916641235 sec
Time for epoch 20 is 463.8230164051056 sec
Time for epoch 25 is 573.0831859111786 sec
Time for epoch 30 is 682.4027009010315 sec
Time for epoch 35 is 791.8056902885437 sec
Time for epoch 40 is 901.0587015151978 sec
Time for epoch 45 is 1010.3610157966614 sec
Time for epoch 50 is 1119.7957785129547 sec
Time for epoch 55 is 1229.104262828827 sec
Time for epoch 60 is 1338.393720626831 sec
Time for epoch 65 is 1447.8509747982025 sec
Time for epoch 70 is 1557.1250584125519 sec
Time for epoch 75 is 1666.414666891098 sec
Time for epoch 80 is 1775.8781249523163 sec
Time for epoch 85 is 1885.1747574806213 sec
Time for epoch 90 is 1994.445820093155 sec
Time for epoch 95 is 2103.8681786060333 sec
Time for epoch 100 is 2213.1362516880035 sec
Time for epoch 105 is 2322.419891834259 sec
Time for epoch 110 is 2431.861670732498 sec
Time for epoch 115 is 2541.1522200107574 sec
Discriminator_config = {
"CNN_1":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":False,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_2":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_3":{"Units": 256,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)}
}
Generator_config = {
"Dense":{"Units":512,"input_shape": (100,),"use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_1":{"Units": 256,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_2":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_3":{"Units": 3,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias": False,
"Batch_norm":False,"Activation":"tanh","last_layer":True}
}
train_dataset,train_x = load_training_data()
## Linear Activation in the final layer of discriminator which then followed by Wassertein loss function
discriminator = make_discriminator_model(layer_config = Discriminator_config,activation="linear")
generator = make_generator_model(layer_config = Generator_config ,dataset = "SVNH")
## RMSProp with smaller learning rate
generator_optimizer,discriminator_optimizer = optimizer("RMSProp",learning_rate = 5e-5)
train(train_dataset,GAN_type = "WGAN",epochs = 120,image_dataset = "SVNH")
training data shape (73257, 32, 32, 3) Model: "sequential_36" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_64 (Conv2D) (None, 16, 16, 64) 4864 _________________________________________________________________ leaky_re_lu_176 (LeakyReLU) (None, 16, 16, 64) 0 _________________________________________________________________ conv2d_65 (Conv2D) (None, 8, 8, 128) 204928 _________________________________________________________________ batch_normalization_85 (Batc (None, 8, 8, 128) 512 _________________________________________________________________ leaky_re_lu_177 (LeakyReLU) (None, 8, 8, 128) 0 _________________________________________________________________ conv2d_66 (Conv2D) (None, 4, 4, 256) 819456 _________________________________________________________________ batch_normalization_86 (Batc (None, 4, 4, 256) 1024 _________________________________________________________________ leaky_re_lu_178 (LeakyReLU) (None, 4, 4, 256) 0 _________________________________________________________________ flatten_18 (Flatten) (None, 4096) 0 _________________________________________________________________ dense_34 (Dense) (None, 1) 4097 ================================================================= Total params: 1,034,881 Trainable params: 1,034,113 Non-trainable params: 768 _________________________________________________________________ Model: "sequential_37" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_35 (Dense) (None, 8192) 819200 _________________________________________________________________ reshape_16 (Reshape) (None, 4, 4, 512) 0 _________________________________________________________________ batch_normalization_87 (Batc (None, 4, 4, 512) 2048 _________________________________________________________________ leaky_re_lu_179 (LeakyReLU) (None, 4, 4, 512) 0 _________________________________________________________________ conv2d_transpose_55 (Conv2DT (None, 8, 8, 256) 3276800 _________________________________________________________________ batch_normalization_88 (Batc (None, 8, 8, 256) 1024 _________________________________________________________________ leaky_re_lu_180 (LeakyReLU) (None, 8, 8, 256) 0 _________________________________________________________________ conv2d_transpose_56 (Conv2DT (None, 16, 16, 128) 819200 _________________________________________________________________ batch_normalization_89 (Batc (None, 16, 16, 128) 512 _________________________________________________________________ leaky_re_lu_181 (LeakyReLU) (None, 16, 16, 128) 0 _________________________________________________________________ conv2d_transpose_57 (Conv2DT (None, 32, 32, 3) 9600 ================================================================= Total params: 4,928,384 Trainable params: 4,926,592 Non-trainable params: 1,792 _________________________________________________________________
Time for epoch 0 is 126.81191229820251 sec
Time for epoch 5 is 677.4151194095612 sec
Time for epoch 10 is 1228.2120637893677 sec
Time for epoch 15 is 1778.7743401527405 sec
Time for epoch 20 is 2329.2324073314667 sec
Time for epoch 25 is 2879.942839860916 sec
Time for epoch 30 is 3430.6335985660553 sec
Time for epoch 35 is 3981.461432695389 sec
Time for epoch 40 is 4532.141057014465 sec
Time for epoch 45 is 5082.906264543533 sec
Time for epoch 50 is 5633.595089197159 sec
Discriminator_config = {
"CNN_1":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":False,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_2":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)}
}
Generator_config = {
"Dense":{"Units":256,"input_shape": (100,),"use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_1":{"Units": 128,"Kernel":(5,5),"Strides":(1,1),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_2":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_3":{"Units": 1,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias": False,
"Batch_norm":False,"Activation":"tanh","last_layer":True}
}
train_dataset,train_x = load_mnist()
discriminator = make_discriminator_model(layer_config = Discriminator_config,size = (64,28,28,1))
generator = make_generator_model(layer_config = Generator_config,size = (64,28,28,1),dataset = "MNIST")
generator_optimizer,discriminator_optimizer = optimizer("Adam")
train(train_dataset,epochs = 50,dataset = "MNIST")
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz 11493376/11490434 [==============================] - 0s 0us/step training data shape (60000, 28, 28, 1) Model: "sequential_4" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_6 (Conv2D) (None, 14, 14, 64) 1664 _________________________________________________________________ leaky_re_lu_20 (LeakyReLU) (None, 14, 14, 64) 0 _________________________________________________________________ conv2d_7 (Conv2D) (None, 7, 7, 128) 204928 _________________________________________________________________ batch_normalization_10 (Batc (None, 7, 7, 128) 512 _________________________________________________________________ leaky_re_lu_21 (LeakyReLU) (None, 7, 7, 128) 0 _________________________________________________________________ flatten_2 (Flatten) (None, 6272) 0 _________________________________________________________________ dense_4 (Dense) (None, 1) 6273 ================================================================= Total params: 213,377 Trainable params: 213,121 Non-trainable params: 256 _________________________________________________________________ Model: "sequential_5" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_5 (Dense) (None, 12544) 1254400 _________________________________________________________________ reshape_2 (Reshape) (None, 7, 7, 256) 0 _________________________________________________________________ batch_normalization_11 (Batc (None, 7, 7, 256) 1024 _________________________________________________________________ leaky_re_lu_17 (LeakyReLU) (None, 7, 7, 256) 0 _________________________________________________________________ conv2d_transpose_6 (Conv2DTr (None, 7, 7, 128) 819200 _________________________________________________________________ batch_normalization_12 (Batc (None, 7, 7, 128) 512 _________________________________________________________________ leaky_re_lu_18 (LeakyReLU) (None, 7, 7, 128) 0 _________________________________________________________________ conv2d_transpose_7 (Conv2DTr (None, 14, 14, 64) 204800 _________________________________________________________________ batch_normalization_13 (Batc (None, 14, 14, 64) 256 _________________________________________________________________ leaky_re_lu_19 (LeakyReLU) (None, 14, 14, 64) 0 _________________________________________________________________ conv2d_transpose_8 (Conv2DTr (None, 28, 28, 1) 1600 ================================================================= Total params: 2,281,792 Trainable params: 2,280,896 Non-trainable params: 896 _________________________________________________________________
Time for epoch 0 is 9.674693822860718 sec
Time for epoch 5 is 45.78199648857117 sec
Time for epoch 10 is 81.81949734687805 sec
Time for epoch 15 is 117.84295916557312 sec
Time for epoch 20 is 154.0394585132599 sec
Time for epoch 25 is 190.0685555934906 sec
Time for epoch 30 is 226.09036922454834 sec
Time for epoch 35 is 262.236704826355 sec
Time for epoch 40 is 298.2668640613556 sec
Time for epoch 45 is 334.25090622901917 sec
Discriminator_config = {
"CNN_1":{"Units": 32,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":False,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_2":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_3":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)}
}
Generator_config = {
"Dense":{"Units":128,"input_shape": (100,),"use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_1":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_2":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_3":{"Units": 1,"Kernel":(5,5),"Strides":(1,1),"Padding":"same","use_bias": False,
"Batch_norm":False,"Activation":"tanh","last_layer":True}
}
train_dataset,train_x = load_mnist()
discriminator = make_discriminator_model(layer_config = Discriminator_config,size = (64,28,28,1),activation="linear")
generator = make_generator_model(layer_config = Generator_config,size = (64,28,28,1),dataset = "MNIST")
generator_optimizer,discriminator_optimizer = optimizer("RMSProp",learning_rate = 5e-5)
train(train_dataset,epochs = 50,GAN_type = "WGAN",image_dataset = "MNIST")
training data shape (60000, 28, 28, 1) Model: "sequential_10" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_16 (Conv2D) (None, 14, 14, 32) 832 _________________________________________________________________ leaky_re_lu_52 (LeakyReLU) (None, 14, 14, 32) 0 _________________________________________________________________ conv2d_17 (Conv2D) (None, 7, 7, 64) 51264 _________________________________________________________________ batch_normalization_23 (Batc (None, 7, 7, 64) 256 _________________________________________________________________ leaky_re_lu_53 (LeakyReLU) (None, 7, 7, 64) 0 _________________________________________________________________ conv2d_18 (Conv2D) (None, 4, 4, 128) 204928 _________________________________________________________________ batch_normalization_24 (Batc (None, 4, 4, 128) 512 _________________________________________________________________ leaky_re_lu_54 (LeakyReLU) (None, 4, 4, 128) 0 _________________________________________________________________ flatten_5 (Flatten) (None, 2048) 0 _________________________________________________________________ dense_10 (Dense) (None, 1) 2049 ================================================================= Total params: 259,841 Trainable params: 259,457 Non-trainable params: 384 _________________________________________________________________ Model: "sequential_11" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_11 (Dense) (None, 6272) 627200 _________________________________________________________________ reshape_5 (Reshape) (None, 7, 7, 128) 0 _________________________________________________________________ batch_normalization_25 (Batc (None, 7, 7, 128) 512 _________________________________________________________________ leaky_re_lu_49 (LeakyReLU) (None, 7, 7, 128) 0 _________________________________________________________________ conv2d_transpose_15 (Conv2DT (None, 14, 14, 128) 409600 _________________________________________________________________ batch_normalization_26 (Batc (None, 14, 14, 128) 512 _________________________________________________________________ leaky_re_lu_50 (LeakyReLU) (None, 14, 14, 128) 0 _________________________________________________________________ conv2d_transpose_16 (Conv2DT (None, 28, 28, 64) 204800 _________________________________________________________________ batch_normalization_27 (Batc (None, 28, 28, 64) 256 _________________________________________________________________ leaky_re_lu_51 (LeakyReLU) (None, 28, 28, 64) 0 _________________________________________________________________ conv2d_transpose_17 (Conv2DT (None, 28, 28, 1) 1600 ================================================================= Total params: 1,244,480 Trainable params: 1,243,840 Non-trainable params: 640 _________________________________________________________________
Time for epoch 0 is 68.2631607055664 sec
Time for epoch 5 is 325.83391070365906 sec
Time for epoch 10 is 583.4614136219025 sec
Time for epoch 15 is 841.4348459243774 sec
Time for epoch 20 is 1099.4572031497955 sec
Time for epoch 25 is 1357.2146186828613 sec
Time for epoch 30 is 1614.990149974823 sec
Time for epoch 35 is 1872.7055423259735 sec
Time for epoch 40 is 2130.7616844177246 sec
Time for epoch 45 is 2388.3034207820892 sec
Generator_config = {
"Dense":{"Units":512,"input_shape": (100,),"use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_1":{"Units": 256,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_2":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_3":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_4":{"Units": 3,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias": False,
"Batch_norm":False,"Activation":"sigmoid","last_layer":True}
}
Discriminator_config = {
"CNN_1":{"Units": 32,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":False,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_2":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_3":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_4":{"Units": 256,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)}
}
train_dataset,train_x = load_celeb_data()
## Linear Activation in the final layer of discriminator which then followed by Wassertein loss function
discriminator = make_discriminator_model(layer_config = Discriminator_config,
size = (64,64,64,3),add_dropout = True)
generator = make_generator_model(layer_config = Generator_config, size = (64,64,64,3),dataset="celeb")
## RMSProp with smaller learning rate
generator_optimizer,discriminator_optimizer = optimizer("Adam")
train(train_dataset,epochs = 200,image_dataset = "celeb",activation = "sigmoid")
training data shape is (10000, 64, 64, 3) 5 real images in the dataset
Model: "sequential_20" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_35 (Conv2D) (None, 32, 32, 32) 2432 _________________________________________________________________ leaky_re_lu_111 (LeakyReLU) (None, 32, 32, 32) 0 _________________________________________________________________ conv2d_36 (Conv2D) (None, 16, 16, 64) 51264 _________________________________________________________________ batch_normalization_44 (Batc (None, 16, 16, 64) 256 _________________________________________________________________ leaky_re_lu_112 (LeakyReLU) (None, 16, 16, 64) 0 _________________________________________________________________ conv2d_37 (Conv2D) (None, 8, 8, 128) 204928 _________________________________________________________________ batch_normalization_45 (Batc (None, 8, 8, 128) 512 _________________________________________________________________ leaky_re_lu_113 (LeakyReLU) (None, 8, 8, 128) 0 _________________________________________________________________ conv2d_38 (Conv2D) (None, 4, 4, 256) 819456 _________________________________________________________________ batch_normalization_46 (Batc (None, 4, 4, 256) 1024 _________________________________________________________________ leaky_re_lu_114 (LeakyReLU) (None, 4, 4, 256) 0 _________________________________________________________________ flatten_10 (Flatten) (None, 4096) 0 _________________________________________________________________ dropout_4 (Dropout) (None, 4096) 0 _________________________________________________________________ dense_18 (Dense) (None, 1) 4097 ================================================================= Total params: 1,083,969 Trainable params: 1,083,073 Non-trainable params: 896 _________________________________________________________________ Model: "sequential_21" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_19 (Dense) (None, 8192) 819200 _________________________________________________________________ reshape_8 (Reshape) (None, 4, 4, 512) 0 _________________________________________________________________ batch_normalization_47 (Batc (None, 4, 4, 512) 2048 _________________________________________________________________ leaky_re_lu_103 (LeakyReLU) (None, 4, 4, 512) 0 _________________________________________________________________ conv2d_transpose_26 (Conv2DT (None, 8, 8, 256) 3276800 _________________________________________________________________ batch_normalization_48 (Batc (None, 8, 8, 256) 1024 _________________________________________________________________ leaky_re_lu_104 (LeakyReLU) (None, 8, 8, 256) 0 _________________________________________________________________ conv2d_transpose_27 (Conv2DT (None, 16, 16, 128) 819200 _________________________________________________________________ batch_normalization_49 (Batc (None, 16, 16, 128) 512 _________________________________________________________________ leaky_re_lu_105 (LeakyReLU) (None, 16, 16, 128) 0 _________________________________________________________________ conv2d_transpose_28 (Conv2DT (None, 32, 32, 64) 204800 _________________________________________________________________ batch_normalization_50 (Batc (None, 32, 32, 64) 256 _________________________________________________________________ leaky_re_lu_106 (LeakyReLU) (None, 32, 32, 64) 0 _________________________________________________________________ conv2d_transpose_29 (Conv2DT (None, 64, 64, 3) 4800 ================================================================= Total params: 5,128,640 Trainable params: 5,126,720 Non-trainable params: 1,920 _________________________________________________________________
Time for epoch 0 is 8.750441312789917 sec
Time for epoch 5 is 35.69864749908447 sec
Time for epoch 10 is 62.64767265319824 sec
Time for epoch 15 is 89.82585000991821 sec
Time for epoch 20 is 116.78434991836548 sec
Time for epoch 25 is 143.7378273010254 sec
Time for epoch 30 is 170.66541862487793 sec
Time for epoch 35 is 197.5900375843048 sec
Time for epoch 40 is 224.76785469055176 sec
Time for epoch 45 is 251.6956911087036 sec
Time for epoch 50 is 278.50433373451233 sec
Time for epoch 55 is 305.26863956451416 sec
Time for epoch 60 is 332.27559304237366 sec
Time for epoch 65 is 359.1018009185791 sec
Time for epoch 70 is 386.0321078300476 sec
Time for epoch 75 is 412.93221521377563 sec
Time for epoch 80 is 439.8767104148865 sec
Time for epoch 85 is 467.0319981575012 sec
Time for epoch 90 is 493.99606347084045 sec
Time for epoch 95 is 520.8957991600037 sec
Time for epoch 100 is 547.8264031410217 sec
Time for epoch 105 is 575.0366084575653 sec
Time for epoch 110 is 601.9921736717224 sec
Time for epoch 115 is 628.9426934719086 sec
Time for epoch 120 is 655.8996784687042 sec
Time for epoch 125 is 682.8523542881012 sec
Time for epoch 130 is 710.0699155330658 sec
Time for epoch 135 is 737.0121128559113 sec
Time for epoch 140 is 763.9644401073456 sec
Time for epoch 145 is 790.8877384662628 sec
Time for epoch 150 is 818.0627119541168 sec
Time for epoch 155 is 845.0173585414886 sec
Time for epoch 160 is 871.964677810669 sec
Time for epoch 165 is 898.93292927742 sec
Time for epoch 170 is 925.8798773288727 sec
Time for epoch 175 is 953.028571844101 sec
Time for epoch 180 is 979.9460124969482 sec
Time for epoch 185 is 1006.8829607963562 sec
Time for epoch 190 is 1033.8327922821045 sec
Time for epoch 195 is 1061.0272171497345 sec
Discriminator_config = {
"CNN_1":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":False,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)},
"CNN_2":{"Units": 128,"Kernel":(5,5),"Strides":(2,2),"Padding":"same",
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2)}
}
Generator_config = {
"Dense":{"Units":256,"input_shape": (100,),"use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_1":{"Units": 128,"Kernel":(5,5),"Strides":(1,1),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_2":{"Units": 64,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias":False,
"Batch_norm":True,"Activation":tf.keras.layers.LeakyReLU(alpha = 0.2),"last_layer":False},
"Transpose_3":{"Units": 1,"Kernel":(5,5),"Strides":(2,2),"Padding":"same","use_bias": False,
"Batch_norm":False,"Activation":"tanh","last_layer":True}
}
train_dataset,train_x = load_mnist()
discriminator = make_discriminator_model(layer_config = Discriminator_config,size = (64,28,28,1))
generator = make_generator_model(layer_config = Generator_config,size = (64,28,28,1),dataset = "MNIST")
generator_optimizer,discriminator_optimizer = optimizer("Adam")
seed = tf.random.normal([16, 100])
start = time.time()
dloss = []
gloss = []
for epoch in range(50):
for image_batch in train_dataset:
noise = tf.random.normal([64, 100])
with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
generated_images = generator(noise, training=True)
real_output = discriminator(image_batch, training=True)
fake_output = discriminator(generated_images, training=True)
gen_loss = generator_loss(fake_output)
disc_loss = discriminator_loss(real_output, fake_output)
gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)
gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)
generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))
discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))
dloss.append(disc_loss)
gloss.append(gen_loss)
plt.figure(figsize=(10, 8))
plt.plot(dloss, label='Discriminitive loss')
plt.plot(gloss, label='Generative loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz 11493376/11490434 [==============================] - 0s 0us/step training data shape (60000, 28, 28, 1) Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d (Conv2D) (None, 14, 14, 64) 1664 _________________________________________________________________ leaky_re_lu (LeakyReLU) (None, 14, 14, 64) 0 _________________________________________________________________ conv2d_1 (Conv2D) (None, 7, 7, 128) 204928 _________________________________________________________________ batch_normalization (BatchNo (None, 7, 7, 128) 512 _________________________________________________________________ leaky_re_lu_1 (LeakyReLU) (None, 7, 7, 128) 0 _________________________________________________________________ flatten (Flatten) (None, 6272) 0 _________________________________________________________________ dense (Dense) (None, 1) 6273 ================================================================= Total params: 213,377 Trainable params: 213,121 Non-trainable params: 256 _________________________________________________________________ Model: "sequential_1" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_1 (Dense) (None, 12544) 1254400 _________________________________________________________________ reshape (Reshape) (None, 7, 7, 256) 0 _________________________________________________________________ batch_normalization_1 (Batch (None, 7, 7, 256) 1024 _________________________________________________________________ leaky_re_lu_2 (LeakyReLU) (None, 7, 7, 256) 0 _________________________________________________________________ conv2d_transpose (Conv2DTran (None, 7, 7, 128) 819200 _________________________________________________________________ batch_normalization_2 (Batch (None, 7, 7, 128) 512 _________________________________________________________________ leaky_re_lu_3 (LeakyReLU) (None, 7, 7, 128) 0 _________________________________________________________________ conv2d_transpose_1 (Conv2DTr (None, 14, 14, 64) 204800 _________________________________________________________________ batch_normalization_3 (Batch (None, 14, 14, 64) 256 _________________________________________________________________ leaky_re_lu_4 (LeakyReLU) (None, 14, 14, 64) 0 _________________________________________________________________ conv2d_transpose_2 (Conv2DTr (None, 28, 28, 1) 1600 ================================================================= Total params: 2,281,792 Trainable params: 2,280,896 Non-trainable params: 896 _________________________________________________________________
<matplotlib.legend.Legend at 0x7fc6e850ca58>